﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.IO;
using System.Net;
using System.Web;
using System.Diagnostics;
using MatchBilingualSent;

namespace CrawlBilingualSentPairFromWeb
{
    public struct SUrl
    {
        public string strUrl;
        public string strRegx;
    }

    public class Crawler
    {
        Queue<SUrl> queueUncrawlUrl = new Queue<SUrl>();

        Dictionary<string, bool> crawledUrl2Regx = new Dictionary<string, bool>();
        Dictionary<string, bool> failedUrlList = new Dictionary<string, bool>();
        HashSet<string> hashCorpus = new HashSet<string>();

        StreamWriter swLog;

        public Crawler()
        {
            swLog = new StreamWriter("Log.txt", true);
        }

        public void LoadCorpus()
        {
            if (File.Exists("Corpus.txt") == true)
            {
                StreamReader sr = new StreamReader("Corpus.txt");
                while (sr.EndOfStream == false)
                {
                    string strLine1, strLine2, strUrl;
                    string strConfidenceScore;

                    strLine1 = sr.ReadLine();
                    strLine2 = sr.ReadLine();
                    strUrl = sr.ReadLine();
                    strConfidenceScore = sr.ReadLine();
                    sr.ReadLine();

                    strLine1 = EasyFilter(strLine1);
                    strLine2 = EasyFilter(strLine2);

                    hashCorpus.Add(strLine1 + strLine2);
                }
                sr.Close();
            }
        }


        public void LoadUrlList()
        {
            queueUncrawlUrl.Clear();
            crawledUrl2Regx.Clear();
            failedUrlList.Clear();

            HashSet<string> seedhash = new HashSet<string>();
            if (File.Exists("SeedUrl.txt") == true)
            {
                StreamReader srSeedUrl = new StreamReader("SeedUrl.txt");
                while (srSeedUrl.EndOfStream == false)
                {
                    SUrl s;
                    s.strUrl = srSeedUrl.ReadLine();
                    s.strRegx = srSeedUrl.ReadLine();

                    queueUncrawlUrl.Enqueue(s);

                    seedhash.Add(s.strUrl);
                    Console.WriteLine("Seed Url:{0}", s.strUrl);
                }
                srSeedUrl.Close();
            }

            if (File.Exists("UncrawlUrl.txt") == true)
            {
                StreamReader srUncrawlUrl = new StreamReader("UncrawlUrl.txt");
                while (srUncrawlUrl.EndOfStream == false)
                {
                    SUrl s;
                    s.strUrl = srUncrawlUrl.ReadLine();
                    s.strRegx = srUncrawlUrl.ReadLine();

                    if (s.strUrl.Contains(" ") == false)
                    {
                        queueUncrawlUrl.Enqueue(s);
                    }
                }
                srUncrawlUrl.Close();
            }

            if (File.Exists("CrawledUrl.txt") == true)
            {
                StreamReader srCrawledUrl = new StreamReader("CrawledUrl.txt");
                while (srCrawledUrl.EndOfStream == false)
                {
                    string line = srCrawledUrl.ReadLine();
                    if (seedhash.Contains(line) == false)
                    {
                        seedhash.Add(line);
                        crawledUrl2Regx.Add(line, true);
                    }
                }
                srCrawledUrl.Close();
            }

            if (File.Exists("FailedUrl.txt") == true)
            {
                StreamReader srFailedUrl = new StreamReader("FailedUrl.txt");
                while (srFailedUrl.EndOfStream == false)
                {
                    string strUrl = srFailedUrl.ReadLine();
                    failedUrlList.Add(strUrl, true);
                }
                srFailedUrl.Close();
            }
        }


        public void SaveUrlList()
        {
            StreamWriter swUncrawlUrl = new StreamWriter("UncrawlUrl.txt");
            foreach (SUrl item in queueUncrawlUrl)
            {
                swUncrawlUrl.WriteLine(item.strUrl);
                swUncrawlUrl.WriteLine(item.strRegx);
            }
            swUncrawlUrl.Close();

            StreamWriter swCrawledUrl = new StreamWriter("CrawledUrl.txt");
            foreach (KeyValuePair<string, bool> pair in crawledUrl2Regx)
            {
                swCrawledUrl.WriteLine(pair.Key);
            }
            swCrawledUrl.Close();

            StreamWriter swFailedUrl = new StreamWriter("FailedUrl.txt");
            foreach (KeyValuePair<string, bool> pair in failedUrlList)
            {
                swFailedUrl.WriteLine(pair.Key);
            }
            swFailedUrl.Close();
        }


        public string EasyFilter(string str)
        {
            while (str.Contains("  ") == true)
            {
                str = str.Replace("  ", " ");
            }

            return str.Trim();
        }

        public void Process()
        {
            ulong ulCount = 0;
            LoadCorpus();

            StreamWriter sw = new StreamWriter("Corpus.txt", true);
            MultiSplit ms = new MultiSplit();
            MatchSent match = new MatchSent();
            WordSeg.WordSeg wordseg = new WordSeg.WordSeg();
            wordseg.LoadLexicalDict("ChineseDictionary.txt");
            match.Initialize(wordseg, 0.18);

            LoadUrlList();

            SUrl s;
            while (queueUncrawlUrl.Count > 0)
            {
                s = queueUncrawlUrl.Dequeue();

                if (crawledUrl2Regx.ContainsKey(s.strUrl) == true || failedUrlList.ContainsKey(s.strUrl) == true)
                {
                    continue;
                }

                try
                {
                    Console.WriteLine("{0}: {1}", ulCount, s.strUrl);

                    string strCont = CrawlerPage(s);

                    strCont = Utility.Formatpage(strCont);
                    List<SentenceToken> stList;
                    List<SentencePair> spList;

                    if (Utility.filter(strCont) == true)
                    {
                        stList = ms.DoMultiSplit(strCont);
                        spList = match.Match(stList);
                        foreach (SentencePair sp in spList)
                        {
                            if (sp.strLine1.Split(' ').Length <= 10 && sp.confidence_score <= 0.30)
                            {
                                continue;
                            }

                            string strLine1, strLine2;

                            strLine1 = EasyFilter(sp.strLine1);
                            strLine2 = EasyFilter(sp.strLine2);

                            string str = strLine1 + strLine2;

                            if (hashCorpus.Contains(str) == false)
                            {
                                hashCorpus.Add(str);

                                sw.WriteLine(sp.strLine1);
                                sw.WriteLine(sp.strLine2);
                                sw.WriteLine(sp.confidence_score);
                                sw.WriteLine(s.strUrl);
                                sw.WriteLine();

                                sw.Flush();


                                Console.WriteLine(sp.strLine1);
                                Console.WriteLine(sp.strLine2);
                                Console.WriteLine(sp.confidence_score);
                                Console.WriteLine(s.strUrl);
                            }
                        }
                    }
                }
                catch (System.Exception err)
                {
                    swLog.WriteLine("Exception:");
                    swLog.WriteLine("URL:{0}", s.strUrl);
                    swLog.WriteLine("Message:{0}", err.Message);
                    swLog.WriteLine("Stack:{0}", err.StackTrace);
                    swLog.WriteLine("----------------------------");

                    swLog.Flush();
                }

                if (crawledUrl2Regx.ContainsKey(s.strUrl) == false)
                {
                    crawledUrl2Regx.Add(s.strUrl, true);
                }

                ulCount++;
                if (ulCount % 20000 == 0)
                {
                    Console.WriteLine("Building New Indexing Data...");

                    sw.Close();

                    SaveUrlList();
                    LoadUrlList();

                    CallExtProgram();

                    sw = new StreamWriter("Corpus.txt", true);
                }
            }

            sw.Close();
            swLog.Close();

            CallExtProgram();
        }


        public void CallExtProgram()
        {
            Process proc = System.Diagnostics.Process.Start("updateDailyData.bat");
            if (proc != null)
            {
                while (true)
                {
                    proc.WaitForExit(3000);
                    if (proc.HasExited)
                    {
                        return;
                    }
                }
            }
        }

        public string MergeUrl(string strUrl, string strRootUrl)
        {

            if (strUrl.Length == 0)
            {
                return "";
            }

            if (strUrl.Length >= 7 && strUrl.Trim().Substring(0, 7).ToLower() == "http://")
            {
                //独立的url, 不需要做整合
                return strUrl;
            }


            Uri ub = new Uri(strRootUrl);
            if (ub.AbsolutePath.Length > 0 && ub.AbsolutePath[0] != '/')
            {
                if (strUrl[0] != '/')
                {
                    strUrl = "/" + strUrl;
                }
            }


            Uri u = new Uri(ub, strUrl);
            return u.ToString();
        }

        public string ProcessYeeyan(SUrl s)
        {
            string strCont = "";

            s.strUrl = s.strUrl.Replace("view", "compare");
            try
            {
                WebClient wc = new WebClient();

                if (s.strUrl.Contains("yeeyan") == true)
                {
                    wc.Headers.Add("Cookie", "CAKEPHP=87a0ff3927556ce67c31d9a28ce605eb; __gads=ID=008ab7132f64de54:T=1269665106:S=ALNI_Mb0LSomFHDWYc_TssZzVb3qOOEkHg; __utma=68569166.426431760.1269665108.1269790150.1269792181.5; __utmz=68569166.1269790364.4.4.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=%E8%80%8C%E6%98%AF%E4%B8%BA%E4%BA%86%E5%9C%A8%E7%BA%BD%E7%BA%A6%E5%B8%82%E6%88%96%E6%B4%9B%E6%9D%89%E7%9F%B6%E4%B9%B0%E4%B8%80%E4%B8%AA%E7%99%BE%E4%B8%87%E7%BE%8E%E5%85%83%E7%9A%84%E8%B1%AA%E5%AE%85; __utmb=68569166.4.10.1269792181; __utmc=68569166; CakeCookie[ykey]=19988; cnzz_a1907081=25; sin1907081=http%3A//www.google.com.hk/search%3Fhl%3Dzh-CN%26newwindow%3D1%26safe%3Dstrict%26q%3D%25E8%2580%258C%25E6%2598%25AF%25E4%25B8%25BA%25E4%25BA%2586%25E5%259C%25A8%25E7%25BA%25BD%25E7%25BA%25A6%25E5%25B8%2582%25E6%2588%2596%25E6%25B4%259B%25E6%259D%2589%25E7%259F%25B6%25E4%25B9%25B0%25E4%25B8%2580%25E4%25B8%25AA%25E7%2599%25BE%25E4%25B8%2587%25E7%25BE%258E%25E5%2585%2583%25E7%259A%2584%25E8%25B1%25AA%25E5%25AE%2585%26btnG%3DGoogle+%25E6%2590%259C%25E7%25B4%25A2%26meta%3D%26aq%3Df%26aqi%3D%26aql%3D%26oq%3D%26gs_rfai%3D; rtime=1; ltime=1269793439265; cnzz_eid=24989706-1269664509-http%3A//www.yeeyan.org/");
                }

                wc.Credentials = CredentialCache.DefaultCredentials;
                // wc.UseDefaultCredentials = true;
                Byte[] pageData = wc.DownloadData(new Uri(s.strUrl, UriKind.RelativeOrAbsolute));

                strCont = Encoding.UTF8.GetString(pageData);

                if (strCont.Contains("charset=GB2312") == true)
                {
                    strCont = Encoding.GetEncoding("GB2312").GetString(pageData);
                }

                wc.Dispose();
            }
            catch (System.Exception err)
            {
                Console.WriteLine(err.Message);
                failedUrlList.Add(s.strUrl, true);
                return strCont;
            }


            Match m = Regex.Match(strCont, "<span>原文</span></h2>([\\s|\\S|]*?)</div>");
            string strHZ = "", strENG = "";
            List<string> strENGList = new List<string>();
            if (m.Success == true)
            {
                strENG = m.Groups[1].ToString().Trim();
                m = Regex.Match(strENG, "<p*?>([\\s|\\S|]*?)</p>");
                while (m.Success == true)
                {
                    string ss = m.Groups[1].ToString().Trim();
                    if (ss.Length > 0)
                    {
                        strENGList.Add(ss);
                    }
                    m = m.NextMatch();
                }
            }

            m = Regex.Match(strCont, "<span>译文</span></h2>([\\s|\\S|]*?)</div>");
            List<string> strHZList = new List<string>();
            if (m.Success == true)
            {
                strHZ = m.Groups[1].ToString().Trim();
                m = Regex.Match(strHZ, "<p*?>([\\s|\\S|]*?)</p>");
                while (m.Success == true)
                {
                    string ss = m.Groups[1].ToString().Trim();
                    if (ss.Length > 0)
                    {
                        strHZList.Add(ss);
                    }
                    m = m.NextMatch();
                }
            }

            int i, j;
            strCont = "";
            for (i = 0, j = 0; i < strENGList.Count && j < strHZList.Count; i++, j++)
            {
                strCont = strCont + strENGList[i] + "\r\n" + strHZList[j];
            }

            while (i < strENGList.Count)
            {
                strCont = strCont + "\r\n" + strENGList[i];
                i++;
            }

            while (j < strHZList.Count)
            {
                strCont = strCont + "\r\n" + strHZList[j];
                j++;
            }

            return strCont;
        }

        public string ProcessWSJ(string strCont)
        {
            Match m = Regex.Match(strCont, @"<!content_tag txt>(.+?)<!/content_tag txt>");
            string strHZ = "", strENG = "";
            if (m.Success == true)
            {
                strHZ = m.Groups[1].ToString().Trim();
            }

            m = Regex.Match(strCont, "en_bodytext=\"(.+?)\"");
            if (m.Success == true)
            {
                strENG = m.Groups[1].ToString().Trim();
            }

            strCont = strHZ + "\r\n" + strENG;

            return strCont;


        }

        public string CrawlerPage(SUrl s)
        {
            string strCont = "";
            try
            {
                WebClient wc = new WebClient();

                if (s.strUrl.Contains("yeeyan") == true)
                {
                    wc.Headers.Add("Cookie", "CAKEPHP=87a0ff3927556ce67c31d9a28ce605eb; __gads=ID=008ab7132f64de54:T=1269665106:S=ALNI_Mb0LSomFHDWYc_TssZzVb3qOOEkHg; __utma=68569166.426431760.1269665108.1269790150.1269792181.5; __utmz=68569166.1269790364.4.4.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=%E8%80%8C%E6%98%AF%E4%B8%BA%E4%BA%86%E5%9C%A8%E7%BA%BD%E7%BA%A6%E5%B8%82%E6%88%96%E6%B4%9B%E6%9D%89%E7%9F%B6%E4%B9%B0%E4%B8%80%E4%B8%AA%E7%99%BE%E4%B8%87%E7%BE%8E%E5%85%83%E7%9A%84%E8%B1%AA%E5%AE%85; __utmb=68569166.4.10.1269792181; __utmc=68569166; CakeCookie[ykey]=19988; cnzz_a1907081=25; sin1907081=http%3A//www.google.com.hk/search%3Fhl%3Dzh-CN%26newwindow%3D1%26safe%3Dstrict%26q%3D%25E8%2580%258C%25E6%2598%25AF%25E4%25B8%25BA%25E4%25BA%2586%25E5%259C%25A8%25E7%25BA%25BD%25E7%25BA%25A6%25E5%25B8%2582%25E6%2588%2596%25E6%25B4%259B%25E6%259D%2589%25E7%259F%25B6%25E4%25B9%25B0%25E4%25B8%2580%25E4%25B8%25AA%25E7%2599%25BE%25E4%25B8%2587%25E7%25BE%258E%25E5%2585%2583%25E7%259A%2584%25E8%25B1%25AA%25E5%25AE%2585%26btnG%3DGoogle+%25E6%2590%259C%25E7%25B4%25A2%26meta%3D%26aq%3Df%26aqi%3D%26aql%3D%26oq%3D%26gs_rfai%3D; rtime=1; ltime=1269793439265; cnzz_eid=24989706-1269664509-http%3A//www.yeeyan.org/");
                }

                wc.Credentials = CredentialCache.DefaultCredentials;
               // wc.UseDefaultCredentials = true;
                Byte[] pageData = wc.DownloadData(new Uri(s.strUrl, UriKind.RelativeOrAbsolute));

                strCont = Encoding.UTF8.GetString(pageData);

                if (strCont.Contains("charset=GB2312") == true)
                {
                    strCont = Encoding.GetEncoding("GB2312").GetString(pageData);
                }

                wc.Dispose();
            }
            catch (System.Exception err)
            {
                Console.WriteLine(err.Message);
                failedUrlList.Add(s.strUrl, true);
                return strCont;
            }


            string strPickUpUrl;

            strCont = HttpUtility.HtmlDecode(strCont);

           
            Match m = Regex.Match(strCont, "href=\"(.+?)\"");
            while (m.Success == true)
            {
                strPickUpUrl = m.Groups[1].ToString().Trim().ToLower();

                if (strPickUpUrl.Length >= 7 && strPickUpUrl.Substring(0, 7) == "mailto:")
                {
                    m = m.NextMatch();
                    continue;
                }
                if (strPickUpUrl.Length > 1)
                {
                    try
                    {
                        strPickUpUrl = MergeUrl(strPickUpUrl, s.strUrl);
                        if (Regex.IsMatch(strPickUpUrl, s.strRegx) == true)
                        {
                            //对www.ftchinese.com站点做特殊处理
                            if (strPickUpUrl.Contains("ftchinese") == true)
                            {
                                if (strPickUpUrl.Substring(strPickUpUrl.Length - 3) != "/ce")
                                {
                                    if (strPickUpUrl[strPickUpUrl.Length - 1] == '/')
                                    {
                                        strPickUpUrl = strPickUpUrl + "ce";
                                    }
                                    else
                                    {
                                        strPickUpUrl = strPickUpUrl + "/ce";
                                    }
                                }
                            }

                            if (crawledUrl2Regx.ContainsKey(strPickUpUrl) == false && strPickUpUrl.Contains(" ") == false)
                            {
                                SUrl u;
                                u.strUrl = strPickUpUrl;
                                u.strRegx = s.strRegx;

                                if (queueUncrawlUrl.Contains(u) == false)
                                {
                                    queueUncrawlUrl.Enqueue(u);
                                }
                            }
                        }
                    }
                    catch (System.Exception err)
                    {
                        ;
                    }
                }

                m = m.NextMatch();
            }

            if (s.strUrl.Contains("chinese.wsj.com") == true)
            {
                strCont = ProcessWSJ(strCont);
            }
            else
            {
                if (s.strUrl.Contains("yeeyan") == true)
                {
                    strCont = ProcessYeeyan(s);
                }
            }



            strCont = strCont.Replace("&nbsp;", " ");
            strCont = strCont.Replace("<br>", "\r\n");
            strCont = strCont.Replace("<BR>", "\r\n");
            strCont = strCont.Replace('\r', ' ');
            strCont = strCont.Replace('\n', ' ');
            strCont = strCont.Replace('\t', ' ');

            strCont = Regex.Replace(strCont, @"<script[\s|\S|]*?</script>", "").Trim();
            strCont = Regex.Replace(strCont, @"<style[\s|\S|]*?</style>", "").Trim();
            strCont = Regex.Replace(strCont, @"<.+?>", "").Trim();


            return strCont;

        }
    }
}
